# Process RAW PUBLIC SIGNALS DATA 
rm(list=ls())

(WD <- getwd())
if (!is.null(WD)) setwd(WD)

library(haven)
library(AER)
library(sandwich)
library(lmtest)
library(pracma)
library(stargazer)
library(plm)
library(pracma)
library(DataCombine)
library(jtools)
library(plyr)
library(readxl)
library(foreign)

lagpad <- function(x, k) {
  res <- c(rep(NA, k), x)[1:length(x)]
  return(res)
}

input0 = "/Users/akohl/Dropbox/7 Rational Inattention/_resubmission/_code/_data/_average_data/_spfrawdata/us_spf_pgdp_avr.rda"
input1 = "/Users/akohl/Dropbox/7 Rational Inattention/_resubmission/_code/_data/_individual_data/_otherrawdata/us_liv_cpi_ind.rda"

##### MICH DATA #############
data         = read_excel("mich-fred.xls", sheet = "Data")
data$mich    = as.numeric(data$MICH) 
tmp          = row(data)
tmp          = tmp[,1]
data$qdate   = 1978+(tmp-1)/4

data         = subset(data, select = c(qdate,mich))

save(data,file="raw_data_mich.rda")

##### SCE DATA #############
data         = read_excel("sce-frbny.xlsx", sheet = "Data")
data$sce     = as.numeric(data$`Median one-year ahead expected inflation rate`)
data         = data[seq(1, nrow(data), 3), ]
tmp          = row(data)
tmp          = tmp[,1]
data$qdate   = 2013.25+(tmp-1)/4

data         = subset(data, select = c(qdate,sce))

save(data,file="raw_data_sce.rda")

##### SPF DATA #############
load(input0)
data$spf    = data$f2

data         = subset(data, select = c(qdate,spf))

save(data,file="raw_data_spf.rda")

##### LIV DATA #############
load(input1)

data        = ddply(data, .(qdate), summarize, fc_err = mean(fc_err, na.rm = TRUE), f2 = mean(f2, na.rm = TRUE))
data$liv    = data$f2

data         = subset(data, select = c(qdate,liv))

save(data,file="raw_data_liv.rda")
clear()

##### MERGE #############
load("raw_data_mich.rda")
data_new    = data
data_series =  c('raw_data_sce.rda','raw_data_spf.rda','raw_data_liv.rda')
for ( i in data_series){
  load(i)
  data_new = merge(data_new, data, all=TRUE)
}

data = data_new[data_new$qdate>=1970,]
save(data,file="public_signals_survey.rda")
clear()
